#!/usr/bin/env ruby

require 'rubygems'
require 'hpricot'
require 'net/http'

$USER_AGENT = 'iTunes/10.3.1 (Windows; Microsoft Windows 7 Ultimate Edition Service Pack 1 (Build 7601)) AppleWebKit/533.21.1'


puts 'script started'

def extract_words(text, storage)
    hash = {}
    text.scan(/\w{3,}/){ |m| 
        key = m.to_s.downcase
        #count = storage[key]
        #count = 0 if count.nil?
        #does it matter how many time you mention an anchor word?
        #storage[key] = count + 1
        
        hash[key] = 1
    }
    
    hash.each { |k,v| 
        v0 = storage[k]
        v0 = 0 if v0.nil?
        storage[k] = v0 + v
    }
end

def parse_item(http, raw_item, item)
    index = raw_item.at("//span[@class='index']/span").inner_text[0..-3].to_i
    name = raw_item.at("//ul[@class='list']/li[@class='name']/a").inner_text
    id = raw_item["adam-id"].to_i
    genre = raw_item.at("//ul[@class='list']/li[@class='genre']").inner_text
    
    str_price = raw_item.at("//ul[@class='list']/li/div/div/button/span[@class='price']").inner_text
    str_price = '$0' if str_price == ''
    price = str_price[1..-1].to_f
    
    link = raw_item.at("//ul[@class='list']/li[@class='name']/a")["href"]
    
    puts "fetching details of #{name}..."
    uri = URI.parse(link)
    raw_details = Hpricot(http.get(uri.path, "User-Agent" => $USER_AGENT ).body)
    puts "ok"
    
    raw_desc = raw_details.at("div[@class='product-info']/div/p")
    raw_desc.search("br").each do |br| br.swap("\n") end
    
    full_desc = raw_desc.inner_text.gsub("\t", "    ");
    short_desc = ""
    full_desc.each { |line| short_desc=line.chomp; break; }
    
    item["index"] = index
    item["name"] = name
    item["id"] = id
    item["genre"] = genre
    item["price"] = price
    item["link"] = link
    item["short_desc"] = short_desc
    item["short_desc_length"] = short_desc.length
    item["full_desc"] = full_desc.gsub("\n", ". ")
    item["full_desc_length"] = full_desc.length
end

def parse_list(data_link)

    puts "openning #{data_link}..."

    data_uri = URI.parse(data_link)
    puts "data uri: #{data_uri.host}:#{data_uri.port}#{data_uri.path}?#{data_uri.query}"
    
    Net::HTTP.start( data_uri.host, data_uri.port ) do |http|
        doc = Hpricot(http.get("#{data_uri.path}?#{data_uri.query}", "User-Agent" => $USER_AGENT ).body)
    
        puts "ok"
        
        data_name = doc.at("//div[@class='content']/div/div[@class='title']/h1/a").inner_text
        data_name += "_" + Time.now.localtime.strftime("%Y-%m-%d")
        
        puts "data_name=#{data_name}"
        
        main_storage = {}
=begin        TODO uncomment
        i = 0
        doc.search("//div[@aria-label]").each do |raw_item|
            break if (i+=1) > 25 
            
            item = {}
            parse_item(http, raw_item, item)
            extract_words(item["full_desc"], main_storage)
        end
=end        
        #puts "main_storage=#{main_storage.inspect}"
        
        #sort by count
        sorted_words = main_storage.sort_by { |k,v| v}
        sorted_words = sorted_words.reverse[0..500]
        puts "sorted_words=#{sorted_words.inspect}"
        
        word_columns = []
        sorted_words.each { |pair| word_columns.push(pair[0]) }
        puts "word_colums=#{word_columns.inspect}"
        
        apps = "index\tname\tid\tgenre\tlink\tprice\tshort_desc\tshort_desc_length\tfull_desc\tfull_desc_length" 
        word_columns.each { |col| apps += "\t#{col}" }
        apps += "\n"
        
        puts "columns=#{apps}"
        
        j = 0
        doc.search("//div[@aria-label]").each do |raw_item|
            break if (j+=1) > 1000
            
            item = {}
            parse_item(http, raw_item, item)
            storage = {}
            extract_words(item["full_desc"], storage)
            
            app = "#{item['index']}\t#{item['name']}\t#{item['id']}\t#{item['genre']}\t#{item['link']}"
            app += "\t#{item['price']}\t#{item['short_desc']}\t#{item['short_desc_length']}\t#{item['full_desc']}\t#{item['full_desc_length']}"
  
            word_columns.each { |col| 
                count = storage[col]
                count = 0 if count.nil?
                app += "\t#{count}" 
            }
            
            apps += "#{app}\n"
        end
        
        #puts apps
        
        file_name = "#{data_name}.cvs"
        puts "writing to file #{file_name}..."
        File.open(file_name, 'w') {|f| f.write(apps) }
        puts "ok"
        
    end
end

# entry point

#parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?id=25204&popId=38&genreId=36')
#parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?id=25180&popId=38&genreId=6014')

=begin
#fetch all games
puts "parsing 'top iphone games'..."
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?id=25180&popId=38&genreId=6014')
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?id=25180&popId=30&genreId=6014')
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?id=25180&popId=27&genreId=6014')
puts 'ok'
=end

#primary categories
(6001..6020).each { |genreId|
    begin
    id = 26331 #+ 10 * (genreId - 7000)

    puts "parsing 'top grossing iphone apps' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=38")
    puts 'ok'
    puts "parsing 'top free iphone apps' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=27")
    puts 'ok'
    puts "parsing 'top paid iphone apps' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=30")
    rescue
        puts "parsing genre #{genreId} failed. trying again..."
        retry
    end
}


#game subcategories
(7001..7019).each { |genreId|
    begin
    id = 26331 + 10 * (genreId - 7000)

    puts "parsing 'top grossing iphone games' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=38")
    puts 'ok'
    puts "parsing 'top free iphone games' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=27")
    puts 'ok'
    puts "parsing 'top paid iphone games' (genreId=#{genreId})..."
    parse_list("http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=#{genreId}&id=#{id}&popId=30")
    rescue
        puts "parsing genre #{genreId} failed. trying again..."
        retry
    end
}


#####
=begin
puts "parsing 'top weather iphone apps'..."
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=6001&id=25292&popId=27')
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=6001&id=25292&popId=30')
parse_list('http://itunes.apple.com/WebObjects/MZStore.woa/wa/viewTop?genreId=6001&id=25292&popId=38')
puts 'ok'
=end

puts 'script finished'
